﻿using Common.Msg;
using Common.Data;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using CToolsWin.DAL;
using CToolsWin.Model;
using Common.Http;
using System.Net;
using System.Text;
using HtmlAgilityPack;
using Model;
using CommonData.Config;

namespace CToolsWin.BLL
{
    public class CommonCrawler
    {
        private int saveType = 0;//保存方式：0：数据库；1：文本
        private int requestType = 0;//采集请求类型：0：范围；1：序列；2数据库；3文本
        private Method method = 0;//请求方式：采集请求类型为数据库或文本时固定为Get请求
        private Dictionary<string, string> dicHeaders = null;
        private Dictionary<string, string> dicHeadersParam = null;
        private Dictionary<string, string> dicParams = null;

        public CommonCrawler(int _saveType, int _requestType, Dictionary<string, string> _dicParams, Dictionary<string, string> _dicHeaders = null, Method _method = Method.Get)
        {
            this.saveType = _saveType;
            this.requestType = _requestType;
            this.dicParams = _dicParams;
            this.dicHeaders = _dicHeaders;
            dicHeadersParam = new Dictionary<string, string>();
            dicHeadersParam.Add("timeout", GetDicValue("timeout"));
            dicHeadersParam.Add("contentType", GetDicValue("contentType"));
            dicHeadersParam.Add("host", GetDicValue("host"));
            dicHeadersParam.Add("accept", GetDicValue("accept"));
            dicHeadersParam.Add("connection", GetDicValue("connection"));
            dicHeadersParam.Add("referer", GetDicValue("referer"));
            dicHeadersParam.Add("userAgent", GetDicValue("userAgent"));
            if (_requestType == 2 || _requestType == 3)
                this.method = Method.Get;
            else
                this.method = _method;
        }
        public string GetDicValue(string key)
        {
            if (dicHeaders != null && dicHeaders.ContainsKey(key))
            {
                string val = dicHeaders[key];
                dicHeaders.Remove(key);
                return val;
            }
            return "";
        }

        public string GetHtmlStr(string strUrl)
        {
            int timeOut = 0;
            Int32.TryParse(dicHeadersParam["timeout"], out timeOut);
            HttpHeaders headers = new HttpHeaders();
            headers.timeout = timeOut;
            headers.contentType = dicHeadersParam["contentType"];
            headers.host = dicHeadersParam["host"];
            headers.accept = dicHeadersParam["accept"];
            headers.connection = dicHeadersParam["connection"];
            headers.referer = dicHeadersParam["referer"];
            headers.userAgent = dicHeadersParam["userAgent"];
            if (method == Method.Get)
                return HttpHelper.Get(strUrl, Encoding.UTF8, dicParams, headers, dicHeaders);
            else
                return HttpHelper.Post(strUrl, Encoding.UTF8, dicParams, headers, dicHeaders);
        }
        public void Crawler(List<string> listUrl, string rangeParamKey, List<string> rangeParamValues, List<CrawlerFields> listField, string strConn, string strTable)
        {
            try
            {
                foreach (var item in listUrl)
                {
                    if (string.IsNullOrWhiteSpace(rangeParamKey) == false && rangeParamValues != null && rangeParamValues.Count > 0)
                    {
                        dicParams.Add(rangeParamKey, "");
                        foreach (var paramItem in rangeParamValues)
                        {
                            dicParams[rangeParamKey] = paramItem;
                            StartCrawler(item, listField, strConn, strTable);
                        }
                    }
                    else
                        StartCrawler(item, listField, strConn, strTable);
                }
            }
            catch (Exception ex)
            {
                Log.GetLog.Error("获取内容出现错误", typeof(GrabAddress), ex);
            }
        }
        private void StartCrawler(string strUrl, List<CrawlerFields> listField, string strConn, string strTable)
        {
            //获取html
            string html = GetHtmlStr(strUrl);

            List<Dictionary<string, string>> listData = new List<Dictionary<string, string>>();
            //解析
            listField = listField.Where(s => string.IsNullOrWhiteSpace(s.Node) == false).ToList();
            IEnumerable<CrawlerFields> listFirst = listField.Where(s => s.PathLength == 1);
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            foreach (var item in listFirst)
            {
                listData.AddRange(Analysis(0, doc.DocumentNode, listField));
            }

            //保存
            SaveData(listData, strConn, strTable);
        }

        /// <summary>
        /// 解析
        /// </summary>
        /// <param name="strHtml"></param>
        /// <param name="listField"></param>
        private List<Dictionary<string, string>> Analysis(int times, HtmlNode html, IEnumerable<CrawlerFields> listField)
        {
            times++;
            Dictionary<string, string> dicValue = new Dictionary<string, string>();
            List<Dictionary<string, string>> listRes = new List<Dictionary<string, string>>();

            foreach (var item in listField.Where(s => s.PathLength == times))
            {
                HtmlNodeCollection nodeList = html.SelectNodes(item.Node);
                if (nodeList == null)
                    continue;

                foreach (var nodeItem in nodeList)
                {
                    if (string.IsNullOrWhiteSpace(item.Field) == false)
                    {
                        if (string.IsNullOrWhiteSpace(item.Key))
                            listRes.Add(new Dictionary<string, string>() { { item.Field, nodeItem.InnerText } });
                        else if (nodeItem.Attributes.Contains(item.Key))
                            listRes.Add(new Dictionary<string, string>() { { item.Field, nodeItem.Attributes[item.Key].Value } });
                    }
                    else
                    {
                        List<Dictionary<string, string>> listValue = Analysis(times, HtmlNode.CreateNode(nodeItem.OuterHtml), listField.Where(s => s.PathLength >= times && s.Path.StartsWith(item.Path)).ToList());
                        if (string.IsNullOrWhiteSpace(item.Field) == false)
                        {
                            if (string.IsNullOrWhiteSpace(item.Key))
                                listValue.ForEach(v => v.Add(item.Field, nodeItem.InnerText));
                            else if (nodeItem.Attributes.Contains(item.Key))
                                listValue.ForEach(v => v.Add(item.Field, nodeItem.Attributes[item.Key].Value));
                        }
                        listRes.AddRange(listValue);
                    }
                }
            }
            if (dicValue.Count > 0)
                listRes.Add(dicValue);
            return listRes;
        }

        public void SaveData(List<Dictionary<string, string>> list, string strConn, string strTable)
        {
            CrawlerDAL dal = new CrawlerDAL();
            foreach (var item in list)
            {
                Console.WriteLine(Common.Serialize.JsonHelper.ToJson(item));
                //dal.Insert(item, strTable, strConn);
            }
        }






        #region 获取京东数据之行政区域信息
        private static AddressRepository addressRepository = new AddressRepository();
        /// <summary>
        /// 初始化表
        /// </summary>
        public void InitTable()
        {
            //初始化表
            Console.WriteLine("{0} 地址信息表初始化 - -", DateTime.Now);
            addressRepository.InitTable();
            //获取商品分类信息
            Console.WriteLine("{0} 地址信息开始抓取 - -", DateTime.Now);
            Crawler();
        }
        /// <summary>
        /// 爬虫
        /// </summary>
        public void Crawler()
        {
            try
            {
                List<Address> addressList = new List<Address>() {
                    new Address() { ParentId = 0, UnicodeName = StrHelper.StringToUnicodeByCN("中国"), Code = "1", ParentCode = "0", Name = "中国", CreatedTime = DateTime.Now } ,
                    new Address() { ParentId = 0, UnicodeName = StrHelper.StringToUnicodeByCN("海外"), Code = "2", ParentCode = "0", Name = "海外", CreatedTime = DateTime.Now }
                };
                addressRepository.Save(addressList, 1);
                Console.WriteLine("国家数据初始化完成，共抓取地址{0}个", addressList.Count);

                //获取html
                string html = HttpHelper.Get(SystemConfig.AddressRootUrl);

                //解析
                html = html.Substring(html.IndexOf('(') + 1);
                List<BaseAddress> list = Common.Serialize.JsonHelper.ToObject<List<BaseAddress>>(html.Trim(')'));
                addressList = list.Where(b => b.name != "中国" && b.name != "海外").Select<BaseAddress, Address>((b, a) =>
                {
                    if (Regex.IsMatch(b.name, "([\u4e00-\u9fa5])+"))
                    {
                        return new Address() { ParentId = 1, UnicodeName = StrHelper.StringToUnicodeByCN(b.name), Code = b.ID.ToString(), ParentCode = "0", Name = b.name, CreatedTime = DateTime.Now };
                    }
                    else
                    {
                        return new Address() { ParentId = 2, UnicodeName = b.name, Code = b.ID.ToString(), ParentCode = "0", Name = b.name, CreatedTime = DateTime.Now };
                    }
                }).OrderBy(a => Convert.ToInt32(a.Code)).ToList();
                //插入数据库
                addressRepository.Save(addressList, 2);
                Console.WriteLine("省级地址数据初始化完成，共抓取地址{0}个", addressList.Count);
                addressList = addressRepository.QueryList(2);
                foreach (Address add in addressList)
                {
                    List<Address> cityAddressList = GetAddressList(SystemConfig.AddressUrl, add.Code, add.ID);
                    if (cityAddressList != null)
                        addressRepository.Save(cityAddressList, 3);
                    Console.WriteLine("市级地址数据初始化完成，共抓取地址{0}个", cityAddressList.Count);
                    cityAddressList = addressRepository.QueryList(3, add.Code);

                    foreach (Address areaAdd in cityAddressList)
                    {
                        List<Address> areaAddressList = GetAddressList(SystemConfig.AddressUrl, areaAdd.Code, areaAdd.ID);
                        if (areaAddressList != null)
                            addressRepository.Save(areaAddressList, 4);
                        Console.WriteLine("地级地址数据初始化完成，共抓取地址{0}个", areaAddressList.Count);
                    }
                }
                Console.WriteLine("地址数据初始化完成");
            }
            catch (Exception ex)
            {
                Log.GetLog.Error("获取内容出现错误", typeof(GrabAddress), ex);
            }
        }
        public List<Address> GetAddressList(string url, string jdParentCode, long parentId)
        {
            try
            {
                //获取html
                string html = HttpHelper.Get(url + jdParentCode);
                if (string.IsNullOrEmpty(html))
                {
                    Log.GetLog.Warn("获取内容为空,url=" + url + jdParentCode, typeof(GrabAddress));
                }
                //解析
                html = html.Substring(html.IndexOf('(') + 1);
                List<BaseAddress> list = Common.Serialize.JsonHelper.ToObject<List<BaseAddress>>(html.Trim(')'));
                List<Address> addressList = list.Select<BaseAddress, Address>((b, a) =>
                {
                    return new Address() { ParentId = parentId, UnicodeName = StrHelper.StringToUnicodeByCN(b.name), Code = b.ID.ToString(), ParentCode = jdParentCode, Name = b.name, CreatedTime = DateTime.Now };
                }).OrderBy(a => Convert.ToInt32(a.Code)).ToList();
                if (addressList == null || addressList.Count == 0)
                {
                    Log.GetLog.Warn("获取地址数据为空,url=" + url + parentId, typeof(GrabAddress));
                }
                return addressList;
            }
            catch (Exception ex)
            {
                Log.GetLog.Error("获取内容出现错误", typeof(GrabAddress), ex);
            }
            return null;
        }
        #endregion
    }
}
